# -*- coding: utf-8 -*-
'''
Created on 2011-11-23

@author: Administrator
'''
from sinastock import *
import urllib2
import simplejson
import os
import urllib2
import sqlite3
import datetime
import threading
import Queue
import time
import shutil 
from setting import *

def myfloat(src):
    if len(src.strip())==0:return 0
    if len(src.replace('-','').strip())==0:return 0
#    print src
    return float(src)
class ThreadStockDataDownload(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.queue=urlqueue
    def run(self):
        code=self.queue.get()  
        
        url='http://stockdata.stock.hexun.com/2009_cgjzd_'+code[2:8]+'.shtml'
#        print url
            
        req = urllib2.Request(url)
        req.add_header('host', 'stockdata.stock.hexun.com')
        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:8.0.1) Gecko/20100101 Firefox/8.0.1')
        req.add_header('Accept-Language', 'zh-cn,zh;q=0.5')
        req.add_header('Accept-Encoding', 'gzip, deflate')
        req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')

        try:
            content=urllib2.urlopen(url).read()
        except:
            print '[x]',url
            fname=os.path.join(errordatapath,code)
            f=open(fname,'w')
#            f.write(content)
            f.close()        
            self.queue.task_done()  
            return
        if content[:5]=='Error':
            print '[r]',#url
            fname=os.path.join(errordatapath,code)
            f=open(fname,'w')
#            f.write(content)
            f.close() 
            self.queue.task_done()
            return
        else:
            start=content.find('class="web2">')
            end=content.find('</table',start)
            tlist=content[start:end].split('<tr')
            tlist.pop(0)
            tlist.pop(0)
            tlist.pop(0)
            tlist.pop(len(tlist)-1)
    #        print len(tlist)
            sharelist=[]
            for t in tlist:
                rlist={}
                dlist=t.split('</span')
    #            print dlist[0]
                i=dlist[0].rfind('>',0)
                date='20'+dlist[0][i+1:].replace('年前','0').replace('季','').replace('年第','0').replace('年年度','04').replace('年中期','02')
                rlist['date']=date
                i=dlist[1].rfind('>',0)
                rlist['personnumber']=(dlist[1][i+1:].replace(',',''))
                i=dlist[2].rfind('>',0)
                rlist['personshare']=(dlist[2][i+1:].replace(',',''))
                i=dlist[4].rfind('>',0)
                rlist['allshare']=(dlist[4][i+1:].replace(',',''))
                i=dlist[5].rfind('>',0)
                rlist['marketshare']=(dlist[5][i+1:].replace(',',''))
                sharelist.append(rlist)
                fname=os.path.join(rawcontentpath,code)
                f=open(fname,'w')
                f.write(rlist['date']+'|')
                f.write(rlist['personnumber']+'|')
                f.write(rlist['personshare']+'|')
                f.write(rlist['allshare']+'|')
                f.write(rlist['marketshare']+'\n')
                f.close()  
                filetobedelete=os.path.join(jobtodopath,code)
                if os.path.exists(filetobedelete):
                    try:
                        os.remove(filetobedelete)
                    except:
                        print 'remove error',filetobedelete
                self.queue.task_done()
        self.__stop()

class ThreadMain(threading.Thread):
    def __init__(self,num):
        threading.Thread.__init__(self)
        self.num=num
    def run(self):          
#        print 'g_jobsnumber',self.num
        for i in range(self.num):
            dt=ThreadStockDataDownload()
            dt.start()
        urlqueue.join()

class StockDataDownload():
    def __init__(self):
        if not os.path.exists(rawcontentpath):os.mkdir(rawcontentpath)
        if not os.path.exists(errordatapath):os.mkdir(errordatapath)
        if not os.path.exists(jobtodopath):os.mkdir(jobtodopath)
        if not os.path.exists(stockbycodepath):os.mkdir(stockbycodepath)
        if not os.path.exists(stockbydatepath):os.mkdir(stockbydatepath)
        for reg in reglist:
            dbpath=os.path.join(stockbycodepath,reg)
            if not os.path.exists(dbpath):os.mkdir(dbpath)
#            self.clearpathfile(dbpath)
        for reg in reglist:
            bydatepath=os.path.join(stockbydatepath,reg)
            if not os.path.exists(bydatepath):os.mkdir(bydatepath)
    def clearpathfile(self,mypath):
        for root,dirs,files in os.walk(mypath):
            for f in files:
                os.remove(os.path.join(root,f)) 
    def movepathfile(self,srcpath,dstpath):
        for root,dirs,files in os.walk(srcpath):
            for f in files:
                dstfile=os.path.join(dstpath,f)
                fdst=open(dstfile,'w')
                fdst.close()
                os.remove(os.path.join(root,f))          
    def geterrorjobslist(self):
        errorlist=self.getpathfilelist(errordatapath)
        return errorlist
    def getpathfilelist(self,mypath,filter=''):
        flist=[]
        for root,dirs,files in os.walk(mypath):
            for f in files:
                if filter=='':flist.append(f.replace('_',':'))
                else:
                    if f.find(filter)>0:flist.append(f.replace('_',':'))
        return flist     
   
    def downloadstart(self,stockall):
        def downloadround():
            todolist=[]
            todolist=self.getsomejobs()
            g_jobsnumber=len(todolist) 
    #        print 'this round todo:',len(todolist)
            while g_jobsnumber>0:
                print 'this round todo:',g_jobsnumber,datetime.datetime.now()
                for job in todolist:
                    urlqueue.put(job,block=True ,timeout= threadtimeout)             
                td=ThreadMain(g_jobsnumber)
                td.start()
                td.join(manthreadtimeout)
                todolist=self.getsomejobs()
                g_jobsnumber=len(todolist)   
            errorlist=self.geterrorjobslist()
            return len(errorlist) 
        ############################################           
        self.clearpathfile(rawcontentpath)
        self.clearpathfile(errordatapath)
        self.clearpathfile(jobtodopath)
#        jobs=StockList()
        ######################################
#        print stockall
        jobnum=0
        for reg in stockall:
            for stock in stockall[reg].keys():                
                fname=os.path.join(jobtodopath,reg+stock+'.txt')
                f=open(fname,'w')
                f.close()   
                jobnum=jobnum+1
##################################################
        i=0
        print 'round start',i
        self.movepathfile(errordatapath,jobtodopath)
        errornum=downloadround()
        print '##########',i,jobnum,errornum
        while errornum!=jobnum:
            jobnum=errornum
            i=i+1
            print 'round start',i
            self.movepathfile(errordatapath,jobtodopath)
            errornum=downloadround()
            print '##########',i,jobnum,errornum
##############################################

    def getsomejobs(self):
        todolist=[]
        joblist=self.getpathfilelist(jobtodopath)
        errorlist=self.geterrorjobslist()
        for job in joblist:
            if job not in errorlist:
                todolist.append(job)
        print 'left job:',len(todolist)
        if todolist==[]:return todolist
        else:            
            jobnum=len(todolist)
#            print 'go here',jobnum,workers_number,jobnum<=workers_number
            if jobnum<=workers_number:
                return todolist
            else:
#                print todolist[:workers_number]
                return todolist[:workers_number]

    def checkstockfile(self):
        rawlist=[]
        for root,dirs,files in os.walk(rawcontentpath):
            for f in files:
                rawlist.append(f)
                fname=os.path.join(root,f)
                f=open(fname,'r')
                content=f.read()
                f.close()
                tlist=content.split('\n')
                s=tlist[0].find('Count')
                p=tlist[0].find('=',s)
                q=tlist[0].find('"',p)
                num=int(tlist[0][p+1:q])
                flines=len(tlist)
                if flines==num+3:pass
                else:print 'checkstockfile failed:',len(tlist),num,fname
                
        print 'raw file num:',len(rawlist),rawlist[0]
        dblist=[]
        for reg in reglist:
            rlist=self.getpathfilelist(os.path.join(stockbycodepath,reg))
            for r in rlist:
                p=r.find('.')
                dblist.append(r[:p]+'_'+reg)
        print 'db file num:',len(dblist),dblist[0]
        
        for raw in rawlist:
            if raw in dblist:pass
            else:print raw,'not in dblist'
        return

class ShareHolder():
    def __init__(self):
        if os.path.exists(shareholderpath) ==False:os.mkdir(shareholderpath)
        if os.path.exists(shareholderresultpath) ==False:os.mkdir(shareholderresultpath)
    def getshareholderbycode(self,code):
        url='http://stockdata.stock.hexun.com/2009_cgjzd_'+code+'.shtml'
        req = urllib2.Request(url)
        req.add_header('host', 'stockdata.stock.hexun.com')
        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:8.0.1) Gecko/20100101 Firefox/8.0.1')
        req.add_header('Accept-Language', 'zh-cn,zh;q=0.5')
        req.add_header('Accept-Encoding', 'gzip, deflate')
        req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')

        content=urllib2.urlopen(req).read().decode('gbk')
        start=content.find('class="web2">')
        end=content.find('</table',start)
        tlist=content[start:end].split('<tr')
        tlist.pop(0)
        tlist.pop(0)
        tlist.pop(0)
        tlist.pop(len(tlist)-1)
#        print len(tlist)
        sharelist=[]
        for t in tlist:
            rlist={}
            dlist=t.split('</span')
#            print dlist[0]
            i=dlist[0].rfind('>',0)
            date='20'+dlist[0][i+1:].replace('年前','0').replace('季','').replace('年第','0').replace('年年度','04').replace('年中期','02')
            rlist['date']=date
            i=dlist[1].rfind('>',0)
            rlist['personnumber']=(dlist[1][i+1:].replace(',',''))
            i=dlist[2].rfind('>',0)
            rlist['personshare']=(dlist[2][i+1:].replace(',',''))
            i=dlist[4].rfind('>',0)
            rlist['allshare']=(dlist[4][i+1:].replace(',',''))
            i=dlist[5].rfind('>',0)
            rlist['marketshare']=(dlist[5][i+1:].replace(',',''))
            sharelist.append(rlist)
##############################################
        fname=os.path.join(shareholderpath,code)
        f=open(fname,'w')            
        for rlist in sharelist:
            f.write(rlist['date']+'|')
            f.write(rlist['personnumber']+'|')
            f.write(rlist['personshare']+'|')
            f.write(rlist['allshare']+'|')
            f.write(rlist['marketshare']+'\n')
        f.close()  
        return sharelist
    def getstocklist(self):
        sina=SINAStock()
        stocklist={}
        slist=sina.getstocklist('SH')
        for s in slist.keys():
            stocklist[s]=slist[s]
        slist=sina.getstocklist('SZ')
        for s in slist.keys():
            stocklist[s]=slist[s]
        return stocklist        

    def updateshareholder(self):
        stocklist=self.getstocklist()
        shareholder={}
        for stock in stocklist:
            print stock
            data=self.getshareholderbycode(stock) 
            shareholder[stock]=data
        print len(shareholder.keys())
#        for stock in shareholder.keys():
#            print stock,len(shareholder[stock])
    def loadshareholderfromcode(self,stock):
        filename=os.path.join(shareholderpath,stock)
        if os.path.exists(filename)==False:self.getshareholderbycode(stock)
        f=open(filename,'r')
        content=f.read()
        tlist=content.split('\n')
        result=[]
        for t in tlist:
            blist=t.split('|')
            rlist={}
#            print len(blist)
            if len(blist)==5:
                rlist['date']=int(blist[0])     
                rlist['personnumber']=myfloat(blist[1])
                rlist['personshare']=myfloat(blist[2])
                rlist['allshare']=myfloat(blist[3])
                rlist['marketshare']=myfloat(blist[4])
                result.append(rlist)
        return result
    def shareholderresult2(self):
        result=[]
        stocklist=self.getstocklist()
        shareholder={}
        personmarketrate={}

        for stock in stocklist:
            shareholder[stock]=self.loadshareholderfromcode(stock)
#            print stock,len(shareholder[stock])
            personmarketrate[stock]={}
            if len(shareholder[stock])>1:
                sh=shareholder[stock]                
                for s in sh:
                    if s['marketshare']==0:personmarketrate[stock][s['date']]=0
                    else:personmarketrate[stock][s['date']]=s['personshare']/s['marketshare']
#            print stock#,personmarketrate
#            break
        xlsdict={}
        for stock in personmarketrate.keys():
            for onejidu in personmarketrate[stock]:
#                print 'onejidu=',onejidu,'stock=',stock
                if onejidu in xlsdict.keys():
                    xlsdict[onejidu][stock]=personmarketrate[stock][onejidu]
                else:
                    xlsdict[onejidu]={}
                    xlsdict[onejidu][stock]=personmarketrate[stock][onejidu]
#        print xlsdict
        outfilename=os.path.join(shareholderresultpath,str(datetime.date.today())+'.xls')
        output=open(outfilename,'w')
        dlist=[]
#        print reversed(sorted(xlsdict.keys()))
        for date in reversed(sorted(xlsdict.keys())):
            output.write('\t'+str(date))
            dlist.append(date)
        output.write('\n')    
        print dlist
        
        for stock in sorted(personmarketrate.keys()):
            
            output.write(str(stock)+'\t')
            for date in dlist:
#                print stock,date
                output.write(str(personmarketrate[stock][date])+'\t')
            output.write('\n')  
        output.close()
        print len(result)
        
    def shareholderresult(self):
        result=[]
        stocklist=self.getstocklist()
        shareholder={}
        personnumberrate={}

        for stock in stocklist:
            shareholder[stock]=self.loadshareholderfromcode(stock)
#            print stock,len(shareholder[stock])
            personnumberrate[stock]={}
            if len(shareholder[stock])>1:
                holderlist=shareholder[stock]                
                for r  in range(0,len(holderlist)-1):
                    if holderlist[r+1]==0:pnrate=0
                    else:pnrate=holderlist[r]['personnumber']/holderlist[r+1]['personnumber']
                    shareholder[stock][holderlist[r]['date']]['pnrate']=pnrate
            print stock,stocklist[stock]
            break
#        xlsdict={}
#        for stock in personmarketrate.keys():
#            for onejidu in personmarketrate[stock]:
##                print 'onejidu=',onejidu,'stock=',stock
#                if onejidu in xlsdict.keys():
#                    xlsdict[onejidu][stock]=personmarketrate[stock][onejidu]
#                else:
#                    xlsdict[onejidu]={}
#                    xlsdict[onejidu][stock]=personmarketrate[stock][onejidu]
##        print xlsdict
#        outfilename=os.path.join(shareholderresultpath,str(datetime.date.today())+'.xls')
#        output=open(outfilename,'w')
#        dlist=[]
##        print reversed(sorted(xlsdict.keys()))
#        for date in reversed(sorted(xlsdict.keys())):
#            output.write('\t'+str(date))
#            dlist.append(date)
#        output.write('\n')    
#        print dlist
#        
#        for stock in sorted(personmarketrate.keys()):
#            
#            output.write(str(stock)+'\t')
#            for date in dlist:
##                print stock,date
#                output.write(str(personmarketrate[stock][date])+'\t')
#            output.write('\n')  
#        output.close()
#        print len(result)            
        
if __name__ == '__main__':
    starttime=datetime.datetime.now()
    print 'this is packStockTakeData  tasklist',starttime  
    sh=ShareHolder()
#    sh.getshareholderbycode('000831')
#    sh.updateshareholder()
    sh.shareholderresult()
#    stockall={}
#    sina=SINAStock()
#    stockall['SH']=sina.getstocklist('SH')  
#    stockall['SZ']=sina.getstocklist('SZ')  
#    sdd=StockDataDownload()
#    sh=sdd.downloadstart(stockall)
    
    print 'dostockdb OK',starttime,(datetime.datetime.now()-starttime).seconds    
    pass